import pandas as pd
import numpy as np


path = '../data/'



def generate_all_feature(mode='train'):
    _basic = pd.read_pickle(path + mode+'_basic.pkl')
    _lda = pd.read_pickle(path + mode+'_lda_tfidf.pkl')
    _nmf = pd.read_pickle(path + mode+'_nmf_tfidf.pkl')
    _svd = pd.read_pickle(path + mode+'_svd_tfidf.pkl')
    _pos = pd.read_pickle(path + mode+'_pos.pkl')
    _w2v = pd.read_pickle(path + mode+'_w2v.pkl')

    _fea = np.hstack([_basic,_lda,_nmf,_svd,_pos,_w2v])
    return _fea




train_all = generate_all_feature('train')
valid_all = generate_all_feature('valid')
dev_all = generate_all_feature('dev')

print(train_all.shape,valid_all.shape,dev_all.shape)

pd.to_pickle(train_all,path+'train_X.pkl')
pd.to_pickle(valid_all,path+'valid_X.pkl')
pd.to_pickle(dev_all,path+'dev_X.pkl')